import os
import time

import requests
import parsel
import re
import pdfkit
import datetime

start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(start_time)
config = pdfkit.configuration(wkhtmltopdf=r'E:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe')

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'
}
html_urls = []
for page in range(1, 4):
    print(f'开始爬取第{page}页')
    if page == 1:
        link = 'https://andyoung.blog.csdn.net/category_11326432.html'
    else:
        link = 'https://andyoung.blog.csdn.net/category_11326432_{}.html'.format(page)
    # time.sleep(1)
    response = requests.get(url=link, headers=headers)

    if response.status_code == 200:
        html = response.text
        url_selector = parsel.Selector(html)
    html_urls = url_selector.css('.column_article_list li a::attr(href)').getall()
    print(html_urls)

    for url in html_urls:
        print(f'开始爬取{url}')
        # url = 'https://andyoung.blog.csdn.net/article/details/120028504'
        time.sleep(1)
        response = requests.get(url=url, headers=headers)

        if response.status_code == 200:
            html = response.text
            selector = parsel.Selector(html)
            # 文章作者
            author_url = selector.css('.profile-href::attr(href)').get()
            print(f"作者链接: {author_url}")
            title = selector.css('#articleContentId::text').get()
            new_title = re.sub(r'[\\:*?"<>|/]', '', title).strip()
            content = selector.css('#content_views').get()
            html_str = '''
            <!DOCTYPE html>
            <html lang="en">
            <head>
                <meta charset="UTF-8">
                <title>{title}</title>
            </head>
            <body>
            <a href="{author_url}">原作者博客</a>
            {article}
            </body>
            </html>
            '''
            html_data = html_str.format(author_url=author_url, title=new_title, article=content)
            html_path = f"html/{new_title}.html"
            if os.path.exists(html_path):
                print(f"文件已存在: {html_path}")
                continue
            else:
                print(f"文件不存在: {html_path}")
                with open(html_path, mode='w', encoding='utf-8') as f:
                    f.write(html_data)
                pdf_path = f"pdf/{new_title}.pdf"
                pdfkit.from_file(html_path, pdf_path, configuration=config)
        else:
            print('请求失败')

end_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(end_time)
print('Done!耗时：{}秒'.format((datetime.datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime(
    start_time, '%Y-%m-%d %H:%M:%S')).seconds))
